{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Phase 4. User classification\n",
    "\n",
    "## Contents\n",
    "- [Configuration](#Configuration)\n",
    "  - [Imports](#Imports)\n",
    "  - [Variables](#Variables)\n",
    "  - [Support functions](#Support-functions)\n",
    "- [Users' features](#Users'-features)\n",
    "  - [Manual labels for training purposes](#Manual-labels-for-training-purposes)\n",
    "  - [Map bots to evaluation format](#Map-bots-to-evaluation-format)\n",
    "- [Ensemble Classifier](#Ensemble-Classifier)\n",
    "  - [Preprocessing](#Preprocessing)\n",
    "  - [Evaluation](#Evaluation)\n",
    "  - [Graphical representation](#Graphical-representation)\n",
    "- [Accounts projections](#Accounts-projections)\n",
    "  - [Get humans and bots interactions](#Get-humans-and-bots-interactions)\n",
    "  - [Projection features](#Projection-features)\n",
    "- [Friendship graph](#Friendship-graph)\n",
    "  - [Create nodes csv (Gephi input 1)](#Create-nodes-csv-(Gephi-input-1))\n",
    "  - [Create edges csv (Gephi input 2)](#Create-edges-csv-(Gephi-input-2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "\n",
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports for data management\n",
    "from fastprogress import master_bar, progress_bar\n",
    "import sys\n",
    "from pymongo import MongoClient, UpdateOne\n",
    "import os \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.options.display.max_columns = None\n",
    "pd.options.display.max_rows = None\n",
    "import pickle\n",
    "import csv\n",
    "from collections.abc import MutableMapping\n",
    "from bson import ObjectId\n",
    "\n",
    "# Imports for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(font_scale=0.9)\n",
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_style({'font.family':'monospace'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If true exports vectorial PDFs instead of JPG.\n",
    "VECTORIAL_FIGURES = True\n",
    "FIG_EXTENSION = \".pdf\" if VECTORIAL_FIGURES else \".jpg\"\n",
    "\n",
    "# Directories where CSV data is stored\n",
    "ROOT_DIR = \"ABOSLUTE_PATH_TO_ROOT_FOLDER\"\n",
    "DATA_DIR = ROOT_DIR + \"data/\"\n",
    "GRAPHICS_DIR = ROOT_DIR + \"graphics/\"\n",
    "\n",
    "# Change path to root\n",
    "os.chdir(ROOT_DIR)\n",
    "\n",
    "try:\n",
    "    os.makedirs(GRAPHICS_DIR)\n",
    "except FileExistsError:\n",
    "    # directory already exists\n",
    "    pass\n",
    "\n",
    "# MongoDB parameters\n",
    "mongoclient = MongoClient('IP_ADDRESS', PORT)\n",
    "db = mongoclient.DATABASE_NAME"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_objid(text):\n",
    "    \"\"\"Makes an ObjectId of 4 bytes\n",
    "    \n",
    "    Keyword arguments:\n",
    "    text -- string to be converted into Object ID\n",
    "    \"\"\"\n",
    "    if text is None:\n",
    "        return None\n",
    "    text = str(text)\n",
    "    if not text.strip():\n",
    "        return None\n",
    "    return ObjectId(text.rjust(24,\"0\"))\n",
    "\n",
    "\n",
    "def flatten(d, parent_key='', sep='_'):\n",
    "    \"\"\"Formats MongoDB results\n",
    "    \n",
    "    Keyword arguments:\n",
    "    d -- dictionary with key and uncleaned values\n",
    "    parent_key --\n",
    "    sep --\n",
    "    \"\"\"\n",
    "    items = []\n",
    "    for k, v in d.items():\n",
    "        new_key = parent_key + sep + k if parent_key else k\n",
    "        if isinstance(v, MutableMapping):\n",
    "            items.extend(flatten(v, new_key, sep=sep).items())\n",
    "        else:\n",
    "            items.append((new_key, v))\n",
    "    return dict(items)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Users' features\n",
    "\n",
    "### Manual labels for training purposes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_labeled_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and political party of manually labeled users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    labeled_users = user_collection.find({'political_party':{'$exists':True}},\n",
    "                                  {'_id':1,'political_party':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    labeled_users = list(labeled_users)\n",
    "    print(\"OK; Total labeled users:\", len(labeled_users))\n",
    "    return labeled_users\n",
    "\n",
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"Extracts the ObjectID, tweet type, bag-of-words, sentiment score and user id of all tweets\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.find({},\n",
    "                            {'_id':0,'tweet_type':1,'keywords_summary':1,'sentiment_score':1,'user_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"OK; Total tweets:\", len(tweets))\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "labeled_users = load_labeled_users(db.users)\n",
    "df_labeled_users = pd.DataFrame(labeled_users)\n",
    "display(df_labeled_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_labeled_users.political_party.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_tweets(db.tweets)\n",
    "tweets = [flatten(t) for t in tweets]\n",
    "df_tweets = pd.DataFrame(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get tweets of manually labeled users by joining two DataFrames (df_labeled_users and df_tweets) through 'user_id' key\n",
    "df_tweets = df_tweets[df_tweets['user_id'].isin(df_labeled_users['_id'])]\n",
    "print(\"Tweets associated with labeled users:\",len(df_tweets))\n",
    "df_tweets = df_tweets.join(df_labeled_users.set_index('_id'), on='user_id') ## append political party to interaction\n",
    "before = len(df_tweets)\n",
    "df_tweets.dropna(axis='index',inplace=True)\n",
    "print(\"Invalid interactions with missing information (multimedia content):\",before-len(df_tweets))\n",
    "display(df_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rename columns\n",
    "df_tweets.columns=['tweet_type', 'user_id', 'sentiment','vox', 'pp', 'cs','psoe', 'up',\n",
    "                   'elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh', 'political_party']\n",
    "\n",
    "display(df_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "''' Calculation of the vector of features for each manually labeled user\n",
    "    That is, per tweet type, the average  sentiment  score  towards  any  given  political  party\n",
    "    and  toward  any  subject  thematic in combination  with any political party\n",
    "'''\n",
    "\n",
    "parties = ['vox', 'pp', 'cs', 'psoe', 'up']\n",
    "themes = ['elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh']\n",
    "vectors = []   # vector of features per user\n",
    "\n",
    "# we iterate over the group of interactions of each manually labeled user\n",
    "for user, user_group in df_tweets.groupby('user_id'):\n",
    "    \n",
    "    # add user_id to vector\n",
    "    user_features = {'user_id':user}\n",
    "    \n",
    "    # add initial features; by default, sentiments are set to 0\n",
    "    for typ in ['original','retweet','quote','reply']:\n",
    "        for p in parties:\n",
    "            # tweet type + party theme\n",
    "            user_features[typ+\"-\"+p] = 0.0\n",
    "            for t in themes:\n",
    "                # tweet type + party theme + subject\n",
    "                user_features[typ+\"-\"+p+\"-\"+t] = 0.0\n",
    "    \n",
    "    # add users' political party to vector --> CLASS, target variable for training process\n",
    "    political_party = user_group.political_party.unique()[0]\n",
    "    user_features['political_party'] = political_party\n",
    "    \n",
    "    \n",
    "    # now, complete vector with features\n",
    "\n",
    "    # group interactions of the user per tweet type\n",
    "    for tweet_type, tweet_type_group in user_group.groupby('tweet_type'):\n",
    "        \n",
    "        # group tweet_type interactions per mentioned party\n",
    "        for p in parties:\n",
    "            party_group = tweet_type_group[tweet_type_group[p]==True]\n",
    "            \n",
    "            # only a single party should be mentioned, remaining parties shoud not match\n",
    "            for p2 in parties:\n",
    "                if p2!=p:\n",
    "                    party_group = party_group[party_group[p2]==False]\n",
    "            \n",
    "            # the mean of the sentiments towards a single party (per tweet type) is calculated (without considering possible thematics)\n",
    "            # in other words, the mean(tweet type, party)\n",
    "            without_theme_group = party_group\n",
    "            if without_theme_group.shape[0] > 0:\n",
    "                mean = without_theme_group['sentiment'].mean()\n",
    "                user_features[tweet_type+\"-\"+p] = mean\n",
    "            \n",
    "            # then, despite considering the mention to a party, the mean of the sentiments towards each theme is extracted\n",
    "            # in other words, the mean(tweet type, party, theme) of the user is calculated\n",
    "            for theme in themes:\n",
    "                \n",
    "                # get those tweets mentioning one theme\n",
    "                theme_group = party_group[party_group[theme]==True]\n",
    "                if theme_group.shape[0] > 0:\n",
    "                    mean = theme_group['sentiment'].mean()\n",
    "                    user_features[tweet_type+\"-\"+p+\"-\"+theme] = mean\n",
    "    \n",
    "    # vector of features of the user is kept\n",
    "    vectors.append(user_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vectors of features are converted into a DataFrame\n",
    "df_classifier = pd.DataFrame.from_dict(vectors,orient='columns')\n",
    "display(df_classifier.head(5))\n",
    "df_classifier.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# the training sample is kept with 200 users per political party\n",
    "df_csv = df_classifier.groupby(by=['political_party']).head(200)\n",
    "df_csv.to_csv(DATA_DIR+\"classification/labeled_users_features.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map bots to evaluation format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_bot_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID of bot users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    p95 = 0.6908019160064479\n",
    "    print(\"Query\", end=\" \")\n",
    "    bot_users = user_collection.find({'scores.scores.universal':{'$gte':p95}},\n",
    "                                  {'_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    bot_users = list(bot_users)\n",
    "    print(\"OK; Total bot users:\", len(bot_users))\n",
    "    return bot_users\n",
    "\n",
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"Extracts the ObjectID, tweet type, bag-of-words, sentiment score and user id of all tweets\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.find({},\n",
    "                            {'_id':0,'tweet_type':1,'keywords_summary':1,'sentiment_score':1,'user_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"OK; Total tweets:\", len(tweets))\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bot_users = load_bot_users(db.users)\n",
    "df_bot_users = pd.DataFrame(bot_users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_tweets(db.tweets)\n",
    "tweets = [flatten(t) for t in tweets]\n",
    "df_tweets = pd.DataFrame(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# get bot tweets by joining tweets and bot dataframes through user id\n",
    "df_tweets.dropna(axis='index',inplace=True)\n",
    "print(\"Tweets with sentiment and BoW:\", len(df_tweets))\n",
    "df_tweets = df_tweets[df_tweets['user_id'].isin(df_bot_users['_id'])]\n",
    "print(\"Tweets associated with bots users:\",len(df_tweets))\n",
    "print(\"Bots with valid interactions:\",len(df_tweets.user_id.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rename of columns\n",
    "df_tweets.columns=['tweet_type', 'user_id', 'sentiment','vox', 'pp', 'cs', 'psoe', 'up',\n",
    "                   'elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh']\n",
    "\n",
    "display(df_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "''' Calculation of the vector of features for each bot user\n",
    "    That is, per tweet type, the average  sentiment  score  towards  any  given  political  party\n",
    "    and  toward  any  subject  thematic in combination  with any political party\n",
    "'''\n",
    "\n",
    "parties = ['vox', 'pp', 'cs', 'psoe', 'up']\n",
    "themes = ['elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh']\n",
    "vectors = []      # vector of features per bot\n",
    "\n",
    "# we iterate over the group of interactions of each manually labeled user\n",
    "for user, user_group in df_tweets.groupby('user_id'):\n",
    "    \n",
    "    # we add user id to vector \n",
    "    user_features = {'user_id':user}\n",
    "    \n",
    "    # add initial features; by default, sentiments are set to 0\n",
    "    for typ in ['original','retweet','quote','reply']:\n",
    "        for p in parties:\n",
    "            # tweet type + party theme\n",
    "            user_features[typ+\"-\"+p] = 0.0\n",
    "            for t in themes:\n",
    "                # tweet type + party theme + subject\n",
    "                user_features[typ+\"-\"+p+\"-\"+t] = 0.0\n",
    "               \n",
    "    # now, complete vector with features\n",
    "\n",
    "    # group interactions of the user per tweet type\n",
    "    for tweet_type, tweet_type_group in user_group.groupby('tweet_type'):\n",
    "        \n",
    "        # group tweet_type interactions per mentioned party\n",
    "        for p in parties:\n",
    "            party_group = tweet_type_group[tweet_type_group[p]==True]\n",
    "            \n",
    "            # only a single party should be mentioned, remaining parties shoud not match\n",
    "            for p2 in parties:\n",
    "                if p2!=p:\n",
    "                    party_group = party_group[party_group[p2]==False]\n",
    "                    \n",
    "            # the mean of the sentiments towards a single party (per tweet type) is calculated (without considering possible thematics)\n",
    "            # in other words, the mean(tweet type, party)\n",
    "            without_theme_group = party_group\n",
    "            if without_theme_group.shape[0] > 0:\n",
    "                mean = without_theme_group['sentiment'].mean()\n",
    "                user_features[tweet_type+\"-\"+p] = mean\n",
    "                \n",
    "            # then, despite considering the mention to a party, the mean of the sentiments towards each theme is extracted\n",
    "            # in other words, the mean(tweet type, party, theme) of the user is calculated\n",
    "            for theme in themes:\n",
    "                \n",
    "                # get those tweets mentioning one theme\n",
    "                theme_group = party_group[party_group[theme]==True]\n",
    "                if theme_group.shape[0] > 0:\n",
    "                    mean = theme_group['sentiment'].mean()\n",
    "                    user_features[tweet_type+\"-\"+p+\"-\"+theme] = mean\n",
    "                    \n",
    "    # vector of features of the user is kept\n",
    "    vectors.append(user_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vectors of features are converted into a DataFrame\n",
    "df_classifier = pd.DataFrame.from_dict(vectors,orient='columns')\n",
    "display(df_classifier.head(5))\n",
    "df_classifier.info() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "''' \n",
    "Bots are filtered to consider those with, at least, one direct interaction towards a political party\n",
    "'''\n",
    "\n",
    "def check_direct_mention_political_parties(bot_vector):\n",
    "    ''' Returns True if at least one direct interaction towards a political party exists, False otherwise\n",
    "    \n",
    "    Keywords arguments:\n",
    "    bot_vector -- the vector of features of a bot (a dictionary containing means of sentiments)\n",
    "    '''\n",
    "    \n",
    "    # features regarding direct interaction with political party themes\n",
    "    political_party_interaction_features = ['original-vox','reply-vox','quote-vox','retweet-vox',\n",
    "                                        'original-pp','reply-pp','quote-pp','retweet-pp',\n",
    "                                        'original-cs','reply-cs','quote-cs','retweet-cs',\n",
    "                                        'original-up','reply-up','quote-up','retweet-up',\n",
    "                                        'original-psoe','reply-psoe','quote-psoe','retweet-psoe']\n",
    "    \n",
    "    for ppif in political_party_interaction_features:\n",
    "        # if a mean different from default value is found, the bot is valid for us\n",
    "        if float(bot_vector[ppif]) != 0:\n",
    "            return True\n",
    "        \n",
    "    # all means regarding direct interactions towards party themes are 0 (default value)\n",
    "    # the bot is not valid for us\n",
    "    return False\n",
    "\n",
    "# feature vectors of bots are filtered\n",
    "print(\"Analyzed bot users:\",len(df_classifier))\n",
    "df_classifier_filtered = df_classifier[df_classifier.apply(check_direct_mention_political_parties, axis=1)]\n",
    "df_classifier_filtered.reset_index(inplace=True)\n",
    "print(\"Filtered bot users (at least one directed mention to political parties):\",len(df_classifier_filtered))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bots for prediction are kept\n",
    "df_classifier_filtered.to_csv(DATA_DIR+'classification/bots_features.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Preprocessing of classifier ensemble results, considering the output of six models:\n",
    "Random Forest (RF), Multilayer perceptron (NN), Support Vector Machine (SVM), Naive Bayes (NB), k-Nearest Neighbor (kNN), and AdaBoost (AB)\n",
    "\n",
    "Each row contains the result of a bot classification: the user id, the predicted party per model (that is, the model with more probability) \n",
    "and the probability of belonging to each party per model.\n",
    "'''\n",
    "df_probabilities = pd.read_csv(DATA_DIR+\"classification/bots_probabilities.csv\", low_memory=False)\n",
    "df_probabilities.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_probabilities.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rename of columns\n",
    "models = [\"6. AB\", \"5. kNN\", \"4. NB\", \"3. SVM\", \"2. NN\", \"1. RF\"]\n",
    "parties = ['ciudadanos','pp','psoe','up','vox']\n",
    "rename_columns=['user_id']\n",
    "for model in models:\n",
    "    rename_columns.append(model)\n",
    "for model in models:\n",
    "    for party in parties:\n",
    "        rename_columns.append(model+\"-\"+party)\n",
    "df_probabilities.columns = rename_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "for each user, the mean of the models for each party is calculated\n",
    "we add 5 columns with the name of the parties which contain that mean\n",
    "'''\n",
    "for party in parties:\n",
    "    probabilities = []\n",
    "    for model in models:\n",
    "        probabilities.append(model+\"-\"+party)\n",
    "    df_probabilities[party]=df_probabilities[probabilities].astype(float).mean(1)\n",
    "    \n",
    "df_probabilities.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_political_party(party_probabilities,threshold=4/5):\n",
    "    ''' Gets the predicted political party of a bot. In particular. returns:\n",
    "    a) The polical party with the highest probability if is bigger than threshold\n",
    "    b) The couple of two political parties if the sum of their probabilities is higher than threshold\n",
    "    c) Unknown otherwise, meaning that it is not clearly classified in one or two parties over the threshold\n",
    "    \n",
    "    Keywords arguments:\n",
    "    party_probabilities -- a dictionary with the probabilities of each political party\n",
    "    threshold -- the value to determine the final political inclination depending on a), b) or c)\n",
    "    '''\n",
    "    # parse probabilities\n",
    "    x = {'PP':party_probabilities.pp,\n",
    "         'PSOE':party_probabilities.psoe,\n",
    "         'VOX':party_probabilities.vox,\n",
    "         'UP':party_probabilities.up,\n",
    "         'Ciudadanos':party_probabilities.ciudadanos}\n",
    "    \n",
    "    # the probabilities are sorted in ascending mode\n",
    "    x = {k: v for k, v in sorted(x.items(), key=lambda item: item[1])}\n",
    "\n",
    "    # case a)\n",
    "    if list(x.values())[4] >= threshold:\n",
    "        return list(x.keys())[4]\n",
    "    # case b)\n",
    "    elif list(x.values())[4] + list(x.values())[3] >= threshold:\n",
    "        return list(x.keys())[4]+\"-\"+list(x.keys())[3]\n",
    "    # case c)\n",
    "    else:    \n",
    "        return \"Unclear\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# a new column is added containing the predicted party considering the six models and a confidence threshold\n",
    "df_probabilities['predicted'] = df_probabilities.apply(get_political_party,axis=1,args=(4/5,)).astype('category')\n",
    "df_probabilities.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of bots in each possible combination of prediction\n",
    "df_probabilities.groupby('predicted')['user_id'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save party predictions in MongoDB\n",
    "operations=[]\n",
    "for user_id,political_party in zip(df_probabilities['user_id'],df_probabilities['predicted']):\n",
    "    operations.append(UpdateOne({'_id': make_objid(user_id)}, \n",
    "                 {'$set': {'bot_political_party': political_party}},\n",
    "                 upsert=False\n",
    "                ))\n",
    "    \n",
    "print(\"Inserting political party of\",len(operations),\"bots...\")\n",
    "\n",
    "if len(operations) > 0:\n",
    "    results = db.users.bulk_write(operations)\n",
    "    print(\"M:\", str(results.matched_count).rjust(8, \" \"),\n",
    "              \" I:\", str(results.inserted_count).rjust(8, \" \"),\n",
    "              \" U:\", str(results.upserted_count).rjust(8, \" \"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graphical representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop unclear\n",
    "before = len(df_probabilities)\n",
    "df_probabilities = df_probabilities[df_probabilities.predicted != 'Unclear']   \n",
    "print(\"Bots with unclear ideology:\", before-len(df_probabilities))\n",
    "df_probabilities.predicted.cat.remove_unused_categories(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Plot heatmaps of those bots with one or two predicted political parties. In particular:\n",
    "\n",
    "a) Square heatmap:  summing up the couples partyA-partyB with partyB-partyA \n",
    "b) Triangular heatmap: differenting between partyA-partyB and partyB-partyA\n",
    "'''\n",
    "for triangular in [False,True]:    \n",
    "    \n",
    "    df_heatmap = df_probabilities\n",
    "    \n",
    "    # in triangular version, we exchange factors in some couples to fit them to their homologous\n",
    "    if triangular:\n",
    "        replace = {'VOX-Ciudadanos':'VOX-Ciudadanos',\n",
    "               'Ciudadanos-VOX':'VOX-Ciudadanos',\n",
    "               'PSOE-UP': 'UP-PSOE',\n",
    "               'UP-PSOE': 'UP-PSOE',\n",
    "               'VOX-UP': 'VOX-UP',\n",
    "               'UP-VOX': 'VOX-UP',        \n",
    "               'UP-Ciudadanos': 'UP-Ciudadanos',\n",
    "               'Ciudadanos-UP': 'UP-Ciudadanos',\n",
    "               'VOX-PP':'VOX-PP',\n",
    "               'PP-VOX':'VOX-PP',\n",
    "               'UP-PP':'UP-PP',\n",
    "               'PP-UP':'UP-PP',\n",
    "               'Ciudadanos-PSOE':'PSOE-Ciudadanos',\n",
    "               'PSOE-Ciudadanos':'PSOE-Ciudadanos',\n",
    "               'Ciudadanos-PP':'PP-Ciudadanos',\n",
    "               'PP-Ciudadanos':'PP-Ciudadanos',\n",
    "               'PSOE-VOX':'VOX-PSOE',\n",
    "               'VOX-PSOE':'VOX-PSOE', \n",
    "               'PSOE-PP':'PSOE-PP',\n",
    "               'PP-PSOE':'PSOE-PP'\n",
    "              }\n",
    "        df_heatmap['predicted'] = df_heatmap.predicted.replace(replace)\n",
    "    \n",
    "    # count number of bots per predicted political party\n",
    "    df_heatmap = df_heatmap.groupby('predicted').agg({'user_id':'count'}) #, 'pp':'mean', 'psoe':'mean', 'up':'mean', 'vox':'mean', 'ciudadanos':'mean'})\n",
    "    df_heatmap.columns = ['num_bots']\n",
    "    df_heatmap.reset_index(inplace=True)\n",
    "\n",
    "    # transform single column (party1-party2) in two columns (party1,party2) \n",
    "    df_heatmap[['party1','party2']] = df_heatmap['predicted'].str.split(\"-\",expand=True)\n",
    "    df_heatmap.drop(columns=['predicted'],inplace=True)\n",
    "    df_heatmap.party2.fillna(df_heatmap.party1,inplace=True) # if party2 is none (because bot has only a single political party), copy party1 column in party2 column\n",
    "    \n",
    "    # draw heatmap\n",
    "    ax = sns.heatmap(df_heatmap.pivot(columns='party2', index='party1', values='num_bots'),cmap=sns.light_palette(\"purple\"),annot=True,fmt=\"g\")\n",
    "    bottom, top = ax.get_ylim()\n",
    "    ax.set_ylim(bottom + 0.5, top - 0.5)\n",
    "    if triangular:\n",
    "        ax.set_xlabel(\"Political party\")\n",
    "        ax.set_ylabel(\"Political party\")\n",
    "    else:\n",
    "        ax.set_xlabel(\"Secondary political party\")\n",
    "        ax.set_ylabel(\"Primary political party\")\n",
    "        \n",
    "    ax.set_title(\"Volumes of bots per political parties\")\n",
    "    if triangular:\n",
    "        plt.savefig(GRAPHICS_DIR + \"triangular-filtered-bots-per-political-party\" + FIG_EXTENSION, bbox_inches = \"tight\")\n",
    "    else:\n",
    "        plt.savefig(GRAPHICS_DIR + \"square-filtered-bots-per-political-party\" + FIG_EXTENSION, bbox_inches = \"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accounts projections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_labeled_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and political party of manually labeled users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    labeled_users = user_collection.find({'political_party':{'$exists':True}},\n",
    "                                  {'_id':1,'political_party':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    labeled_users = list(labeled_users)\n",
    "    print(\"OK; Total labeled users:\", len(labeled_users))\n",
    "    return labeled_users\n",
    "\n",
    "def load_bot_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and predicted political partyof bot users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    bot_users = user_collection.find({'bot_political_party':{'$exists':True}},\n",
    "                                  {'_id':1,'bot_political_party':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    bot_users = list(bot_users)\n",
    "    print(\"OK; Total bot users:\", len(bot_users))\n",
    "    return bot_users\n",
    "\n",
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"Extracts the bag-of-words, sentiment score and user id of all tweets\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.find({},\n",
    "                            {'_id':0,'keywords_summary':1,'sentiment_score':1,'user_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"OK; Total tweets:\", len(tweets))\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get humans and bots interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "labeled_users = load_labeled_users(db.users)\n",
    "df_labeled_users = pd.DataFrame(labeled_users)\n",
    "display(df_labeled_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bot_users = load_bot_users(db.users)\n",
    "df_bot_users = pd.DataFrame(bot_users)\n",
    "df_bot_users = df_bot_users[df_bot_users.bot_political_party.isin(['VOX','Ciudadanos','PP','PSOE','UP'])]  # only bots with one predicted party are considered\n",
    "df_bot_users.columns = ['_id','political_party']\n",
    "display(df_bot_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_tweets(db.tweets)\n",
    "tweets = [flatten(t) for t in tweets]\n",
    "df_tweets = pd.DataFrame(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we get the tweets of manually labeled users and append the associated political party to them\n",
    "df_labeled_tweets = df_tweets[df_tweets['user_id'].isin(df_labeled_users['_id'])]\n",
    "print(\"Tweets associated with labeled users:\",len(df_labeled_tweets))\n",
    "df_labeled_tweets = df_labeled_tweets.join(df_labeled_users.set_index('_id'), on='user_id') ## append political party to interaction\n",
    "before = len(df_labeled_tweets)\n",
    "df_labeled_tweets.dropna(axis='index',inplace=True)\n",
    "display(df_labeled_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we get the tweets of bots users and append the predicted political party to them\n",
    "df_bot_tweets = df_tweets[df_tweets['user_id'].isin(df_bot_users['_id'])]\n",
    "print(\"Tweets associated with bot users:\",len(df_bot_tweets))\n",
    "df_bot_tweets = df_bot_tweets.join(df_bot_users.set_index('_id'), on='user_id') ## append political party to interaction\n",
    "before = len(df_bot_tweets)\n",
    "df_bot_tweets.dropna(axis='index',inplace=True)\n",
    "display(df_bot_tweets.head(2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Projection features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_projection_features(df_tweets):\n",
    "    '''\n",
    "    Gets the mean of sentiment towards each political party per user \n",
    "    \n",
    "    Keyword arguments:\n",
    "    df_tweets -- a DataFrame with tweets interactions\n",
    "    '''\n",
    "    parties = ['VOX', 'PP', 'Ciudadanos', 'PSOE', 'UP']\n",
    "    projection_vectors = []\n",
    "    \n",
    "    for user, user_group in df_tweets.groupby('user_id'):\n",
    "        \n",
    "        political_party = user_group.iloc[0]['political_party']   \n",
    "        user_vector = {'user_id':user,\n",
    "                        'political_party':political_party}\n",
    "        \n",
    "        for p in parties:\n",
    "            # by default, the sentiment towards a political party is 0\n",
    "            user_vector[p] = 0.0\n",
    "\n",
    "            # in party_group resides the tweets of user mentioning party, at least, p\n",
    "            party_group = user_group[user_group[p]==True]\n",
    "            \n",
    "            # we filter party_group to remove those tweets that mentions p and another party p2\n",
    "            for p2 in parties:\n",
    "                if p2!=p:\n",
    "                    party_group = party_group[party_group[p2]==False]\n",
    "\n",
    "            # the mean of sentiment of tweets mentioning p is calculated\n",
    "            if len(party_group) > 0:\n",
    "                mean = party_group['sentiment'].mean()\n",
    "                user_vector[p] = mean\n",
    "        \n",
    "    \n",
    "        projection_vectors.append(user_vector)\n",
    "\n",
    "    # we return a DataFrame where each row contains the user id, the political party and five means of sentiment associated with parties\n",
    "    return pd.DataFrame.from_dict(rows,orient='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# we get the feature vectors of bots for projection\n",
    "df_bot_tweets.columns=['user_id', 'sentiment','VOX', 'PP', 'Ciudadanos', 'PSOE', 'UP','elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh','political_party']\n",
    "df_bot_tweets.drop(['elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh'],axis='columns',inplace=True)\n",
    "df_bot_features = get_projection_features(df_bot_tweets)\n",
    "df_bot_features['category'] = 'bot'   # we add the category of 'bot' to these users\n",
    "display(df_bot_features.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# we get the feature vectors of manually labeled users for projection\n",
    "df_labeled_tweets.columns=['user_id','sentiment','VOX','PP', 'Ciudadanos', 'PSOE', 'UP', 'elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh','political_party']\n",
    "df_labeled_tweets.drop(['elections', 'exhumacion', 'cataluña', 'debates', 'abascaleh'],axis='columns',inplace=True)\n",
    "df_labeled_features = get_projection_features(df_labeled_tweets)\n",
    "df_labeled_features['category'] = 'manually labeled'     # we add the category of 'manually labeled' to these users\n",
    "display(df_labeled_features.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Manually labeled users:\",len(df_labeled_features))\n",
    "print(\"Bots users:\",len(df_bot_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bots and manually labeled users are joint before plotting\n",
    "df_unified = df_bot_features.append(df_labeled_features,ignore_index=True)\n",
    "df_unified['political_party'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_unified.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vectors are saved\n",
    "df_unified.to_csv(DATA_DIR+'/classification/projection.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Friendship graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bot_followings(user_collection):\n",
    "    \"\"\"\n",
    "    Extracts the ObjectID, predicted party and followings of bot users with only\n",
    "    one identified political inclination. \n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection - MongoDB Users' collection\n",
    "    \"\"\"\n",
    "    \n",
    "    print(\"Query\", end=\" \")\n",
    "    bot_friendships = user_collection.find(\n",
    "        {'bot_political_party': {'$exists':True},'friends': {'$exists':True}}, \n",
    "        {'_id': 1, 'friends': 1, 'bot_political_party': 1})\n",
    "         \n",
    "    print(\"OK; List\", end=\" \")\n",
    "    bot_friendships = list(bot_friendships) \n",
    "    print(\"OK; Total bots:\", len(bot_friendships), end=\"; \")\n",
    "\n",
    "    df_bots = pd.DataFrame(bot_friendships)\n",
    "    df_bots.columns = ['Id','Followings','Party']\n",
    "    df_bots = df_bots[df_bots.Party.isin(['VOX','UP','Ciudadanos','PP','PSOE'])]\n",
    "    df_bots.set_index('Id',inplace=True)\n",
    "    print(\"Bots of one-affinity party with friends:\", len(df_bots))\n",
    "    \n",
    "    return df_bots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bot_friendships = get_bot_followings(db.users)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create nodes csv (Gephi input 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creation of a CSV with the nodes of the graph. We employ the columns \"ID\" (user id, the identificator) and \"Party\" (political party, the attribute for coloring)\n",
    "bot_friendships[['Party']].to_csv(DATA_DIR+\"gephi/nodes.csv\",index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create edges csv (Gephi input 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "'''Creation of a CSV with the edges of the graph. Each row represents an edge, that is, a particular following\n",
    "Each row has the column \"Source\" (the following user) and \"Target\" (the followed user)\n",
    "'''\n",
    "with open(DATA_DIR+'gephi/edges.csv', 'w') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow(['Source','Target'])\n",
    "    \n",
    "    # we iterate through the bot_friendships DataFrame, considering the followings of each user (user_id)\n",
    "    for user_id,row in progress_bar(list(bot_friendships.iterrows())):\n",
    "        print(\"Processing\", user_id, end=\"; \")\n",
    "        \n",
    "        # we get the followings of the user\n",
    "        followings = pd.DataFrame(row[0],columns=['id'])\n",
    "        print(\"Total followings:\",len(followings), end=\"; \")\n",
    "        \n",
    "        # we filter them to get only those who are bots with only a political party\n",
    "        followings = followings[followings.id.isin(bot_friendships.index)]\n",
    "        print(\"Bot followings:\",len(followings))\n",
    "        \n",
    "        # for each following, a new row in the CSV should be created (that is, an edge)\n",
    "        for following in followings['id']:\n",
    "            writer.writerow([user_id,following])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
